# webapp
from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
import cgi

# database
from google.appengine.ext import db
import database

# do web request
import urllib2

# parse html
import HTMLParser

# json output
from django.utils import simplejson

# for debug
import logging

# total 19 columns
# company_id, company_name
class MopsIncomeStatementParser(HTMLParser.HTMLParser):
	# ctor
	def __init__(self, year, season):
		# parent ctor
		HTMLParser.HTMLParser.__init__(self)
		
		# store the year & season
		self.year = year
		self.season = season
		
		# for counting columns
		self.total_column_count = 0
		self.column_count = 0
	
		# compnay info
		self.company_id = 0
		self.company_name = ""
		
		# income statement instance
		self.income_statement_instance = None
		
		# set process type (0->nothing, 1->header, 2->data)
		self.process_type = 0
		
		# only for debug
		self.debug_msg = ""
		
		# record total parsed record no
		self.parsed_row_no = 0
		
	def handle_starttag(self, tag, attr):
		if self.process_type == 1 and tag == "th": self.process_header()

	def get_company_instance(self):
		# check if need to put company info
		company_instance = None
		company_gql_query = db.GqlQuery("SELECT * FROM company WHERE id=%d LIMIT 1" % self.company_id)
		if company_gql_query.count() > 0: company_instance = company_gql_query.get()
		else:
			company_instance = database.company(id=self.company_id, name=self.company_name.decode("utf-8"))
			company_instance.put()
		return company_instance

	def process_data_normal(self, data):
		# check if instance is none
		if self.income_statement_instance == None: return
		
		# remove comma
		data = data.replace(",", "")
		
		# fill data
		if self.column_count == 3: self.income_statement_instance.operating_revenue = int(data)
		elif self.column_count == 4: self.income_statement_instance.operating_cost = int(data)
		elif self.column_count == 5: self.income_statement_instance.gross_profit = int(data)
		elif self.column_count == 6: self.income_statement_instance.unrealized_gain_on_inter_affiliate_accounts = int(data)
		elif self.column_count == 7: self.income_statement_instance.realized_gain_on_inter_affiliate_accounts = int(data)
		elif self.column_count == 8: self.income_statement_instance.operating_expenses = int(data)
		elif self.column_count == 9: self.income_statement_instance.net_operating_income = int(data)
		elif self.column_count == 10: self.income_statement_instance.non_operating_revenue = int(data)
		elif self.column_count == 11: self.income_statement_instance.non_operating_cost = int(data)
		elif self.column_count == 12: self.income_statement_instance.continuing_operating_income_before_tax = int(data)
		elif self.column_count == 13: self.income_statement_instance.income_tax_expense = int(data)
		elif self.column_count == 14: self.income_statement_instance.continuing_operating_income = int(data)
		elif self.column_count == 15: self.income_statement_instance.discontinued_operating_income = int(data)
		elif self.column_count == 16: self.income_statement_instance.extraordinatry_items = int(data)
		elif self.column_count == 17: self.income_statement_instance.cumulative_effect_of_changes_in_accounting_principles = int(data)
		elif self.column_count == 18: self.income_statement_instance.net_income = int(data)
		elif self.column_count == 19:
			self.income_statement_instance.eps = float(data)
			# complete a whole row
			self.income_statement_instance.put()
			self.income_statement_instance = None
			self.column_count = 0
		else: self.column_count = 0
			
	def process_data(self, data):
		# check which data we are processing
		if self.column_count == 1: # company id
			self.company_id = int(data)
		elif self.column_count == 2: # company name
			self.company_name = data
			
			# get company instance
			company_instance = self.get_company_instance()
			
			# increment parsed row no
			self.parsed_row_no = self.parsed_row_no + 1
			
			# create the income_statement_instance here
			if self.total_column_count == 19:
				self.income_statement_instance = database.income_statement_normal(
					year=self.year, season=self.season, company_ref=company_instance)
			# not defined yet (need to fix in the future)
			else: pass
		else:
			# process based on type
			if self.total_column_count == 19: self.process_data_normal(data)
			# not defined yet (need to fix in the future)
			else: pass

	def process_header(self):
		self.total_column_count = self.total_column_count + 1

	# htmlpaser override function
	def handle_data(self, data):
		if self.process_type == 2:
			# strip data
			data = data.strip()

			# increment column count
			self.column_count = self.column_count + 1
			
			# process data
			self.process_data(data)
		
	def unknown_decl(self, data):
		pass

def retrieve_income_statement( year, season, ret ):
	# check if the statement is already retrieved
	available_instance = None
	available_query = db.GqlQuery("SELECT * FROM mops_data_available WHERE year=%d AND season=%d" % (year, season))
	if available_query.count() > 0: available_instance = company_gql_query.get()
	else: available_query = database.mops_data_available(year=year, season=season,
												income_statement_get=False, balance_sheet_get=False)
	# request the web data
	try:
		req1_post_data = "encodeURIComponent=1&step=1&firstin=1&off=1&TYPEK=sii&year=%d&season=%02d" % (year, season)
		req1 = urllib2.Request(url="http://mops.twse.com.tw/mops/web/ajax_t51sb08", data=req1_post_data)
		req1.add_header("Referer", "http://mops.twse.com.tw/mops/web/t51sb08")
		req1.add_header("Cookie", "newmops2=selfObj%3DtagCon1%7CTYPEK%3Dsii%7Cyear%3D100%7Cseason%3D02%7C")
		f1 = urllib2.urlopen(req1)
		data1 = f1.read()
		f1.close()
	except:
		ret["status"] = "fail"
		ret["error_msg"] = "download data failed. please retry again or check if the mops interface changed."
		return -1
		
	# use parsert to parse the data
	parser = MopsIncomeStatementParser(year, season)
	#try:
	for line in data1.splitlines():
		# parse data
		if line.find("<tr class='tblHead'><th class='tblHead' nowrap>") >= 0:
			parser.total_column_count = 0
			parser.process_type = 1
			parser.feed( line )
		elif line.find("<tr class='cColor'><td align='center'>") >= 0 or line.find("<tr class='lColor'><td align='center'>") >= 0:
			parser.process_type = 2
			parser.feed( line )
	#except:
	#	ret["status"] = "fail"
	#	ret["error_msg"] = "parse mops data exception"
	#	return -1

	# check if parsed any data
	if parser.parsed_row_no <= 0:
		ret["status"] = "fail"
		ret["error_msg"] = "data not avaiable"
		return -3
	else:
		# update mops_data_available
		available_query.income_statement_get = True
		available_query.put()

	return 0


class DataRetrieveHandler(webapp.RequestHandler):
	# when get, show the control ui
	def get(self):
		context = {}
		self.response.out.write(template.render("template/data_retrieve.html", context))

	# process query command
	def process_query(self, ret):
		results = db.GqlQuery("SELECT * FROM mops_data_available")
		for row in results:
			row_data = {}
			for key in row.__dict__["_entity"].keys():
				row_data[key] = row.__dict__["_entity"][key]
			ret["result"].append( row_data )
		
	# process retrieve command	
	def process_retrieve(self, ret):
		year = int(cgi.escape(self.request.get("year")))
		season = int(cgi.escape(self.request.get("season")))
		retrieve_income_statement(year, season, ret)

	# when post, show the result
	def post(self):
		# reset return data
		ret = { "status": "success", "error_msg": "", "result": [] }
		
		# process data by command
		command = cgi.escape(self.request.get("command"))
		
		# query
		if command == "query": self.process_query(ret)
		# retrieve
		elif command == "retrieve": self.process_retrieve(ret)

		# return result
		self.response.out.write( simplejson.dumps(ret) )
